Inspect Modelling Performance

suppressPackageStartupMessages(library(tidyverse))
library(targets)
library(DT)
knitr::opts_knit$set(root.dir = "../../")

VIF

df_cv_preds_and_coefs <- tar_read(df_cv_preds_and_coefs)

df_coefs <- df_cv_preds_and_coefs %>%
  select(year_start_act, category_id, pred) %>%
  mutate(coefs = map(pred, "coefs")) %>%
  select(- pred) %>%
  unnest(coefs) %>%
  unnest(coefs)

df_coefs %>%
  filter(year_start_act == 2019) %>%
  group_by(category_id) %>%
  summarise(max_vif = max(vif, na.rm = TRUE)) %>%
  knitr::kable()
category_id max_vif
cnsn 1.253876
dtin 1.105846
ptpe 1.048755
sfty 1.966747
spno 1.045658

Perfermance Metrics

df_perf <- tar_read(df_perf)

AUC

df_perf %>%
  filter(.metric == "roc_auc") %>%
  knitr::kable(digits = 2)
category_id .metric mean sd n
cnsn roc_auc 0.61 0.15 8
dtin roc_auc 0.60 0.10 8
ptpe roc_auc 0.59 0.06 8
sfty roc_auc 0.63 0.07 8
spno roc_auc 0.53 0.06 8

Brier

df_perf %>%
  filter(.metric == "brier") %>%
  knitr::kable(digits = 2)
category_id .metric mean sd n
cnsn brier 0.24 0.03 8
dtin brier 0.19 0.04 8
ptpe brier 0.23 0.04 8
sfty brier 0.25 0.03 8
spno brier 0.24 0.03 8

Calibration

Linear Calibration

tar_read(df_calib) %>%
  select(- plot_data) %>%
  knitr::kable(digits = 3)
category_id lower base_rate upper intercept slope
cnsn 0.250 0.455 0.593 0.171 0.603
dtin 0.426 0.726 0.849 0.374 0.506
ptpe 0.482 0.691 0.786 0.482 0.304
sfty 0.242 0.474 0.704 0.193 0.577
spno 0.628 0.637 0.637 0.654 -0.026

Plot

tar_read(p_calib)